{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data: Impermium_verification_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib as mpl\n",
    "import matplotlib.pylab as plt\n",
    "from IPython.display import clear_output\n",
    "\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data load\n",
    "data = pd.read_csv('./data/impermium_verification_labels.csv', delimiter = ',', skiprows = 1,\n",
    "                   names = ['id', 'Insult', 'Date', 'Comment', 'Usage'], usecols=['Insult', 'Comment'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Insult</th>\n",
       "      <th>Comment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>\"like this if you are a tribe fan\"</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>\"you're idiot.......................\"</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>\"I am a woman Babs, and the only \"war on women...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>\"WOW &amp; YOU BENEFITTED SO MANY WINS THIS YEAR F...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>\"haha green me red you now loser whos winning ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>\"\\nMe and God both hate-faggots.\\n\\nWhat's the...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>\"Oh go kiss the ass of a goat....and you DUMMY...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>\"Not a chance Kid, you're wrong.\"</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>\"On Some real Shit FUck LIVE JASMIN!!!\"</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>\"ok but where the hell was it released?you all...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Insult                                            Comment\n",
       "0       0                 \"like this if you are a tribe fan\"\n",
       "1       1              \"you're idiot.......................\"\n",
       "2       1  \"I am a woman Babs, and the only \"war on women...\n",
       "3       1  \"WOW & YOU BENEFITTED SO MANY WINS THIS YEAR F...\n",
       "4       1  \"haha green me red you now loser whos winning ...\n",
       "5       0  \"\\nMe and God both hate-faggots.\\n\\nWhat's the...\n",
       "6       1  \"Oh go kiss the ass of a goat....and you DUMMY...\n",
       "7       0                  \"Not a chance Kid, you're wrong.\"\n",
       "8       0            \"On Some real Shit FUck LIVE JASMIN!!!\"\n",
       "9       1  \"ok but where the hell was it released?you all..."
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEWCAYAAACXGLsWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAQ9UlEQVR4nO3dfayedX3H8fdHKuDDlKeDkbazGDs2NFHxiDicYdY5QGf5QxzEjQ6bNEtwiCwboNkwW7ZBtoyJWUg6QItDkKEJbNMxVmBOHcjBB+RBRkUCxyIcVkARFYvf/XF+Zxza0/5OH859tz3vV3JyX9f397uv63snd/Pp9XCuk6pCkqSted6wG5Ak7foMC0lSl2EhSeoyLCRJXYaFJKnLsJAkdRkW0k6S5P4kb5+h/mtJ7hlGT9LOYlhIc6yq/quqDuvNS/LRJP84iJ6kbWVYSJK6DAtp53pdktuTPJHkM0n2TXJMkvGpCUnOSvK9JD9Mck+SZUmOBT4M/HaSJ5N8c3gfQdrcgmE3IO1h3gscC/wE+DLwe8C3pwaTHAZ8AHhjVa1PsgTYq6q+k+QvgVdV1e8MummpxyMLaee6sKrWV9UG4J+B120y/gywD3B4kudX1f1V9Z2BdyltI8NC2rm+P235KeDF0werah1wBvBR4JEkVyY5ZHDtSdvHsJAGrKo+XVVvAV4BFHD+1NDwupK2zrCQBijJYUnelmQfJq9r/JjJU1MADwNLkvjvUrscv5TSYO0DnAc8yuQpq4OZvAsK4J/a6/8m+doQepO2KP7xI0lSj0cWkqQuw0KS1GVYSJK6DAtJUtecPe4jyaXAu4BHquo1rfbXwG8BTwPfAU6tqsfb2DnASiZvIzy9qq5r9WOBjwF7ARdX1Xm9fR900EG1ZMmSnf6ZJGlPdttttz1aVSMzjc3Z3VBJ3go8CVw2LSzeAdxQVRuTnA9QVWclORy4AjgSOAT4D+CX2qb+B/gNYBy4FTi5qu7a2r5HR0drbGxsDj6VJO25ktxWVaMzjc3Zaaiq+iKwYZPav1fVxrZ6M7CoLS8Hrqyqn1bVd4F1TAbHkcC6qrqvqp4GrmxzJUkDNMxrFu8HvtCWFwIPThsbb7Ut1TeTZFWSsSRjExMTc9CuJM1fQwmLJB8BNgKXT5VmmFZbqW9erFpdVaNVNToyMuMpN0nSdhr437NIsoLJC9/L6tkLJuPA4mnTFgHr2/KW6pKkARnokUW7s+ks4N1V9dS0oWuBk5Lsk+RQYCnwVSYvaC9NcmiSvYGT2lxJ0gDN5a2zVwDHAAe1Pyl5LnAOkw9Suz4JwM1V9ftVdWeSq4C7mDw9dVpVPdO28wHgOiZvnb20qu6cq54lSTPbIx8k6K2zkrTthnLrrCRpz2FYSJK6Bn431O5gydn/OuwWtJu6/7x3DrsFaU54ZCFJ6jIsJEldhoUkqcuwkCR1GRaSpC7DQpLUZVhIkroMC0lSl2EhSeoyLCRJXYaFJKnLsJAkdRkWkqQunzor7YJ88rG211w9+dgjC0lSl2EhSeoyLCRJXYaFJKnLsJAkdRkWkqQuw0KS1GVYSJK6DAtJUpdhIUnqmrOwSHJpkkeS3DGtdkCS65Pc2173b/UkuTDJuiS3Jzli2ntWtPn3JlkxV/1KkrZsLo8sPgkcu0ntbGBtVS0F1rZ1gOOApe1nFXARTIYLcC7wJuBI4NypgJEkDc6chUVVfRHYsEl5ObCmLa8BTphWv6wm3Qzsl+TlwG8C11fVhqp6DLiezQNIkjTHBn3N4mVV9RBAez241RcCD06bN95qW6pvJsmqJGNJxiYmJnZ645I0n+0qF7gzQ622Ut+8WLW6qkaranRkZGSnNidJ892gw+LhdnqJ9vpIq48Di6fNWwSs30pdkjRAgw6La4GpO5pWANdMq5/S7oo6Cniinaa6DnhHkv3bhe13tJokaYDm7C/lJbkCOAY4KMk4k3c1nQdclWQl8ABwYpv+eeB4YB3wFHAqQFVtSPLnwK1t3p9V1aYXzSVJc2zOwqKqTt7C0LIZ5hZw2ha2cylw6U5sTZK0jXaVC9ySpF2YYSFJ6jIsJEldhoUkqcuwkCR1GRaSpC7DQpLUZVhIkroMC0lSl2EhSeoyLCRJXYaFJKnLsJAkdRkWkqQuw0KS1GVYSJK6DAtJUpdhIUnqMiwkSV2GhSSpy7CQJHUZFpKkLsNCktRlWEiSugwLSVKXYSFJ6jIsJEldQwmLJB9KcmeSO5JckWTfJIcmuSXJvUk+k2TvNneftr6ujS8ZRs+SNJ8NPCySLAROB0ar6jXAXsBJwPnABVW1FHgMWNneshJ4rKpeBVzQ5kmSBmhYp6EWAC9IsgB4IfAQ8Dbg6ja+BjihLS9v67TxZUkywF4lad4beFhU1feAvwEeYDIkngBuAx6vqo1t2jiwsC0vBB5s793Y5h+46XaTrEoylmRsYmJibj+EJM0zwzgNtT+TRwuHAocALwKOm2FqTb1lK2PPFqpWV9VoVY2OjIzsrHYlSQznNNTbge9W1URV/Qz4HPCrwH7ttBTAImB9Wx4HFgO08ZcCGwbbsiTNb8MIiweAo5K8sF17WAbcBdwIvKfNWQFc05avbeu08RuqarMjC0nS3BnGNYtbmLxQ/TXgW62H1cBZwJlJ1jF5TeKS9pZLgANb/Uzg7EH3LEnz3YL+lJ2vqs4Fzt2kfB9w5AxzfwKcOIi+JEkz8ze4JUldhoUkqcuwkCR1GRaSpC7DQpLUZVhIkroMC0lSl2EhSeoyLCRJXYaFJKnLsJAkdRkWkqQuw0KS1GVYSJK6DAtJUpdhIUnqMiwkSV2GhSSpy7CQJHUZFpKkLsNCktRlWEiSugwLSVKXYSFJ6jIsJEldhoUkqWsoYZFkvyRXJ/l2kruTvDnJAUmuT3Jve92/zU2SC5OsS3J7kiOG0bMkzWfdsEhy+Ay1Y3Zwvx8D/q2qfhl4LXA3cDawtqqWAmvbOsBxwNL2swq4aAf3LUnaRrM5srgqyVntf/gvSPJx4K+2d4dJXgK8FbgEoKqerqrHgeXAmjZtDXBCW14OXFaTbgb2S/Ly7d2/JGnbzSYs3gQsBr4C3AqsB47egX2+EpgAPpHk60kuTvIi4GVV9RBAez24zV8IPDjt/eOtJkkakNmExc+AHwMvAPYFvltVP9+BfS4AjgAuqqrXAz/i2VNOM8kMtdpsUrIqyViSsYmJiR1oT5K0qdmExa1MhsUbgbcAJye5egf2OQ6MV9Utbf1qJsPj4anTS+31kWnzF097/yImj26eo6pWV9VoVY2OjIzsQHuSpE3NJixWVtWfVtXPqur7VbUcuGZ7d1hV3wceTHJYKy0D7gKuBVa02opp+7gWOKVdMzkKeGLqdJUkaTAW9CZU1dgMtU/t4H7/ALg8yd7AfcCpTAbXVUlWAg8AJ7a5nweOB9YBT7W5kqQB6obFXKiqbwCjMwwtm2FuAafNeVOSpC3yN7glSV2GhSSpy7CQJHUZFpKkLsNCktRlWEiSugwLSVKXYSFJ6jIsJEldhoUkqcuwkCR1GRaSpC7DQpLUZVhIkroMC0lSl2EhSeoyLCRJXYaFJKnLsJAkdRkWkqQuw0KS1GVYSJK6DAtJUpdhIUnqMiwkSV2GhSSpy7CQJHUNLSyS7JXk60n+pa0fmuSWJPcm+UySvVt9n7a+ro0vGVbPkjRfDfPI4oPA3dPWzwcuqKqlwGPAylZfCTxWVa8CLmjzJEkDNJSwSLIIeCdwcVsP8Dbg6jZlDXBCW17e1mnjy9p8SdKADOvI4u+APwZ+3tYPBB6vqo1tfRxY2JYXAg8CtPEn2vznSLIqyViSsYmJibnsXZLmnYGHRZJ3AY9U1W3TyzNMrVmMPVuoWl1Vo1U1OjIyshM6lSRNWTCEfR4NvDvJ8cC+wEuYPNLYL8mCdvSwCFjf5o8Di4HxJAuAlwIbBt+2JM1fAz+yqKpzqmpRVS0BTgJuqKr3ATcC72nTVgDXtOVr2zpt/Iaq2uzIQpI0d3al37M4CzgzyTomr0lc0uqXAAe2+pnA2UPqT5LmrWGchvp/VXUTcFNbvg84coY5PwFOHGhjkqTn2JWOLCRJuyjDQpLUZVhIkroMC0lSl2EhSeoyLCRJXYaFJKnLsJAkdRkWkqQuw0KS1GVYSJK6DAtJUpdhIUnqMiwkSV2GhSSpy7CQJHUZFpKkLsNCktRlWEiSugwLSVKXYSFJ6jIsJEldhoUkqcuwkCR1GRaSpC7DQpLUZVhIkroGHhZJFie5McndSe5M8sFWPyDJ9Unuba/7t3qSXJhkXZLbkxwx6J4lab4bxpHFRuAPq+pXgKOA05IcDpwNrK2qpcDatg5wHLC0/awCLhp8y5I0vw08LKrqoar6Wlv+IXA3sBBYDqxp09YAJ7Tl5cBlNelmYL8kLx9w25I0rw31mkWSJcDrgVuAl1XVQzAZKMDBbdpC4MFpbxtvtU23tSrJWJKxiYmJuWxbkuadoYVFkhcDnwXOqKofbG3qDLXarFC1uqpGq2p0ZGRkZ7UpSWJIYZHk+UwGxeVV9blWfnjq9FJ7faTVx4HF096+CFg/qF4lScO5GyrAJcDdVfW304auBVa05RXANdPqp7S7oo4Cnpg6XSVJGowFQ9jn0cDvAt9K8o1W+zBwHnBVkpXAA8CJbezzwPHAOuAp4NTBtitJGnhYVNWXmPk6BMCyGeYXcNqcNiVJ2ip/g1uS1GVYSJK6DAtJUpdhIUnqMiwkSV2GhSSpy7CQJHUZFpKkLsNCktRlWEiSugwLSVKXYSFJ6jIsJEldhoUkqcuwkCR1GRaSpC7DQpLUZVhIkroMC0lSl2EhSeoyLCRJXYaFJKnLsJAkdRkWkqQuw0KS1GVYSJK6DAtJUtduExZJjk1yT5J1Sc4edj+SNJ/sFmGRZC/g74HjgMOBk5McPtyuJGn+2C3CAjgSWFdV91XV08CVwPIh9yRJ88aCYTcwSwuBB6etjwNvmj4hySpgVVt9Msk9A+ptvjkIeHTYTeyqcv6wO9A22CO/yzv4HXzFlgZ2l7DIDLV6zkrVamD1YNqZv5KMVdXosPuQdpTf5W2zu5yGGgcWT1tfBKwfUi+SNO/sLmFxK7A0yaFJ9gZOAq4dck+SNG/sFqehqmpjkg8A1wF7AZdW1Z1Dbmu+8lSf9hR+l7dBqqo/S5I0r+0up6EkSUNkWEiSugwLzZqPXNGeIMmlSR5Jcsewe9mdGBaaFR+5oj3IJ4Fjh93E7saw0Gz5yBXtEarqi8CGYfexuzEsNFszPXJl4ZB6kTRghoVmq/vIFUl7LsNCs+UjV6R5zLDQbPnIFWkeMyw0K1W1EZh65MrdwFU+ckW7oyRXAP8NHJZkPMnKYfe0O/BxH5KkLo8sJEldhoUkqcuwkCR1GRaSpC7DQpLUZVhIu5gkZyR54bD7kKbz1llpF5PkfmC0qh4ddi/SFI8spO2Q5JQktyf5ZpJPJXlFkrWttjbJL7Z5n0zynmnve7K9HpPkpiRXJ/l2kssz6XTgEODGJDcm2att444k30ryoeF8Ys13C4bdgLS7SfJq4CPA0VX1aJIDgDXAZVW1Jsn7gQuBEzqbej3waiafsfXltr0Lk5wJ/Hrb9huAhVX1mrbv/eboY0lb5ZGFtO3eBlw9dZqoqjYAbwY+3cY/BbxlFtv5alWNV9XPgW8AS2aYcx/wyiQfT3Is8IMdbV7aHoaFtO1C//HsU+Mbaf/OkgTYe9qcn05bfoYZjvSr6jHgtcBNwGnAxdvVsbSDDAtp260F3pvkQIB2GuorTD6JF+B9wJfa8v3AG9rycuD5s9j+D4FfaNs+CHheVX0W+BPgiJ3Qv7TNvGYhbaOqujPJXwD/meQZ4OvA6cClSf4ImABObdP/AbgmyVeZDJkfzWIXq4EvJHkIOAP4RJKp/9idsxM/ijRr3jorSeryNJQkqcuwkCR1GRaSpC7DQpLUZVhIkroMC0lSl2EhSer6P/SlRc80dy0oAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "counts, bins = np.histogram(data['Insult'])\n",
    "\n",
    "y = counts\n",
    "x = bins[:-1]\n",
    "xlabel = [0, 1]\n",
    "plt.title(\"hist\")\n",
    "plt.bar(x, y)\n",
    "plt.xticks(xlabel)\n",
    "plt.yticks()\n",
    "plt.xlabel(\"counts\")\n",
    "plt.ylabel(\"x\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['\"like this if you are a tribe fan\"',\n",
       "       '\"you\\'re idiot.......................\"',\n",
       "       '\"I am a woman Babs, and the only \"war on women\" I see is coming from jackazzes like you.\\\\xa0 I don\\'t need your protection or your ignorant rhetoric masquerading as representing my best interests.\\\\xa0 \"',\n",
       "       ...,\n",
       "       '\"sweetie pie is looking very much like her cousin Maggie E...WHAT A BEAUTY THIS LITTLE ONE IS..Love you all to pieces !!\"',\n",
       "       '\"ball4real where are you with your miami g-ayness\"',\n",
       "       '\"Man....if you are a 3 point shooter, you must love playing with Wade and Bron. \"'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['Comment'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(data['Comment'].values, data['Insult'].values, test_size=0.2, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1788,)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['\"Of course you would say that\"',\n",
       "       '\"Go back into hiding Bush... America still hasn\\'t gotten over your failed presidency\"',\n",
       "       '\"Look again. This isn\\'t about the control room when *spoilers* you kill guilty spark.. its when you do the warthog run... it\\'s not \"false\". When the pannels begin to fall, you are at the silent cartographer\\'s island.\"',\n",
       "       '\"Are you and Orochimaru bu.tt buddies too? Disgusting\"',\n",
       "       '\"@Fuck:disqus\\\\xa0@cunt:disqus\\\\xa0@shit:disqus\\\\xa0\\\\n\\\\n@Fuck:disqus\\\\xa0you CNN, I know you certainly have a lot to do with the untimely ends of several of our fellow commenters. \\\\xa0You tried many times to get me, but I\\'m always a step ahead of you.\"',\n",
       "       '\"How would you like to call that mom!\"',\n",
       "       '\"No sir. I love black men and wouldnt trade them for the world. Always treated with respect and always knew how to pick them well cause my mother and father taught me well. No kids btw and currently studying. But ignorant black men like you just fuck society up. Downing black women to reassure yourself that you as an avg black man represents some idyllic shit and black women fail in comparison. Im a good ass woman and can make any man happy preferably black but if he is Latin, White or Asian then thats what it will be. \\\\n\\\\nI thought your comment was ignorance, decided to give you a chance to state your argument effectively but you couldnt hold your own so you failed. Blah\"',\n",
       "       '\"you are a disgrace to dodger fans\"',\n",
       "       '\"@Youknowwhatiiitis: you afro-american negro\\'s only & not blacks from different cultures.\"',\n",
       "       '\"Hey man, glad to hear you\\'re not in serious health straights - as I get \\\\nolder, I worry more and more about the internet people who I follow and \\\\nenjoy to hear from. \\\\n\\\\n\\\\n\\\\nCFS is a motherfucker, and people don\\'t give it enough credit to how \\\\nmuch it can fuck a life up. Fight the good fight man, and if you\\'re ever\\\\n in Australia I\\'ll buy you your carbonated beverage of preference :)\\\\n\"'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(447,)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['\"Are you ever going to show us some cock or butt?\"',\n",
       "       '\"You may be right.\\\\xa0\"',\n",
       "       '\"EVELYN WE NEW YOU WERE TRASH IN THE BEGINNING BUT AS THE SHOW GREW YOU GOT OUT OF CONTROL AND REALLY SHOWED THE WORLD WHO YOU REALLY ARE TRASH. TRASH IS WHAT YOU ARE AND YOU AND CHAD ARE DESERVING OF EACH OTHER! YOU ALL ARE FROM THE SAME FLOCK!\"',\n",
       "       '\"I\\'m guessing English is not your first language. He was being sarcastic.\"',\n",
       "       '\"Pretend you\\'re a gentleman for a little while and grow up little man.  The adults are talking...\"',\n",
       "       '\"Notice they were listed as good manners...\\\\n\\\\n..for children.\\\\n\\\\nWhich leaves me way in the fucking clear. Fuck Erasmus, anyway.\\\\xa0 Preachy old fuck. \"',\n",
       "       '\"Girls need to learn when to shut up, she is beautiful, she knows it and that\\'s all that really matters you don\\'t know if she unhealthy, be quiet with your words of hate we (the majority of women) are tired if the hate.\"',\n",
       "       '\"Racist?  Seriously?  How stupid are you, exactly?\"',\n",
       "       '\"You are so lucky! \"',\n",
       "       '\"Hey, Einstein, perhaps you should have read and understood what you quoted, since it clearly outlines a war room to blunt any movement post-DNC, not during.\\\\n\\\\nIs it really shocking that the GOP would have a war room in NC, a swing state? \\\\n\\\\nReading is fundamental. \"'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(447,)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 1, 0, 1, 0, 1, 1, 0, 1])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test[:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preprocessing for BERT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!pip install ktrain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "using Keras version: 2.2.4-tf\n"
     ]
    }
   ],
   "source": [
    "import ktrain\n",
    "from ktrain import text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- IMDB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# (pre_X_train, pre_y_train), (pre_X_test, pre_y_test), preproc = text.texts_from_folder('./data/aclImdb', \n",
    "#                                                                        maxlen=500, \n",
    "#                                                                        preprocess_mode='bert',\n",
    "#                                                                        train_test_names=['train', \n",
    "#                                                                                          'test'],\n",
    "#                                                                        classes=['pos', 'neg'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Impermium_verification_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_seq_length = 100\n",
    "max_features=8000"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- References: https://github.com/amaiya/ktrain/blob/master/ktrain/text/data.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "preprocessing train...\n",
      "language: en\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "done."
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "preprocessing test...\n",
      "language: en\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "done."
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "(pre_X_train, pre_y_train), (pre_X_test, pre_y_test), preproc = text.texts_from_array(X_train, y_train, X_test, y_test, \n",
    "                   class_names = [0, 1],\n",
    "                   max_features=max_features, maxlen=max_seq_length, \n",
    "                   val_pct=0.1, ngram_range=1, preprocess_mode='bert', \n",
    "                   lang=None, # auto-detected\n",
    "                   random_state=42,\n",
    "                   verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2, 1788, 100)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array(pre_X_train).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[array([[ 101, 1000, 1997, ...,    0,    0,    0],\n",
       "        [ 101, 1000, 2175, ...,    0,    0,    0],\n",
       "        [ 101, 1000, 2298, ...,    0,    0,    0],\n",
       "        ...,\n",
       "        [ 101, 1000, 1045, ...,    0,    0,    0],\n",
       "        [ 101, 1000, 8100, ...,    0,    0,    0],\n",
       "        [ 101, 1000, 5292, ...,    0,    0,    0]]),\n",
       " array([[0, 0, 0, ..., 0, 0, 0],\n",
       "        [0, 0, 0, ..., 0, 0, 0],\n",
       "        [0, 0, 0, ..., 0, 0, 0],\n",
       "        ...,\n",
       "        [0, 0, 0, ..., 0, 0, 0],\n",
       "        [0, 0, 0, ..., 0, 0, 0],\n",
       "        [0, 0, 0, ..., 0, 0, 0]])]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pre_X_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1788, 2)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pre_y_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1., 0.],\n",
       "       [1., 0.],\n",
       "       [1., 0.],\n",
       "       ...,\n",
       "       [1., 0.],\n",
       "       [1., 0.],\n",
       "       [0., 1.]], dtype=float32)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pre_y_train"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# BERT (ktrain module)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Is Multi-Label? False\n",
      "maxlen is 100\n",
      "done.\n"
     ]
    }
   ],
   "source": [
    "model = text.text_classifier('bert', (pre_X_train, pre_y_train) , preproc=preproc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "learner = ktrain.get_learner(model, \n",
    "                             train_data=(pre_X_train, pre_y_train), \n",
    "                             val_data=(pre_X_test, pre_y_test), \n",
    "                             batch_size=32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "simulating training for different learning rates... this may take a few moments...\n",
      "Train on 1788 samples\n",
      "Epoch 1/1024\n",
      "1788/1788 [==============================] - 39s 22ms/sample - loss: 0.9826 - acc: 0.5218\n",
      "Epoch 2/1024\n",
      "1788/1788 [==============================] - 34s 19ms/sample - loss: 0.8995 - acc: 0.5218\n",
      "Epoch 3/1024\n",
      "1788/1788 [==============================] - 34s 19ms/sample - loss: 0.7745 - acc: 0.5218\n",
      "Epoch 4/1024\n",
      "1788/1788 [==============================] - 34s 19ms/sample - loss: 0.6962 - acc: 0.5481\n",
      "Epoch 5/1024\n",
      "1788/1788 [==============================] - 34s 19ms/sample - loss: 0.6655 - acc: 0.6191\n",
      "Epoch 6/1024\n",
      "1788/1788 [==============================] - 34s 19ms/sample - loss: 0.6154 - acc: 0.6667\n",
      "Epoch 7/1024\n",
      "1788/1788 [==============================] - 34s 19ms/sample - loss: 0.5272 - acc: 0.7360\n",
      "Epoch 8/1024\n",
      "1788/1788 [==============================] - 34s 19ms/sample - loss: 0.4027 - acc: 0.8255\n",
      "Epoch 9/1024\n",
      "1788/1788 [==============================] - 34s 19ms/sample - loss: 0.2869 - acc: 0.8893\n",
      "Epoch 10/1024\n",
      "1788/1788 [==============================] - 34s 19ms/sample - loss: 0.2004 - acc: 0.9251\n",
      "Epoch 11/1024\n",
      "1788/1788 [==============================] - 34s 19ms/sample - loss: 0.1802 - acc: 0.9413\n",
      "Epoch 12/1024\n",
      "1788/1788 [==============================] - 34s 19ms/sample - loss: 0.1316 - acc: 0.9541\n",
      "Epoch 13/1024\n",
      "1788/1788 [==============================] - 34s 19ms/sample - loss: 0.2384 - acc: 0.9032\n",
      "Epoch 14/1024\n",
      "1788/1788 [==============================] - 34s 19ms/sample - loss: 0.1668 - acc: 0.9424\n",
      "Epoch 15/1024\n",
      "1788/1788 [==============================] - 34s 19ms/sample - loss: 0.2707 - acc: 0.8921\n",
      "Epoch 16/1024\n",
      "1788/1788 [==============================] - 34s 19ms/sample - loss: 0.3818 - acc: 0.8468\n",
      "Epoch 17/1024\n",
      "1344/1788 [=====================>........] - ETA: 8s - loss: 0.7213 - acc: 0.5357\n",
      "\n",
      "done.\n",
      "Please invoke the Learner.lr_plot() method to visually inspect the loss plot to help identify the maximal learning rate associated with falling loss.\n"
     ]
    }
   ],
   "source": [
    "learner.lr_find()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEKCAYAAAAfGVI8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3dd3hUZdrH8e+dTiolCSW0AAm9h6L0poAKi2KvqIuoqCt23de2lrWs67oLCqJiWVTsCCg2eg0d6SG0UEwIIaQSkjzvHzOyAUJIMGfOlPtzXbmc0+bceRzml9OeR4wxKKWU8l1+dheglFLKXhoESinl4zQIlFLKx2kQKKWUj9MgUEopH6dBoJRSPi7A7gKqKjo62jRt2tTuMpRSyqOsXr36sDEmprxlHhcETZs2ZdWqVXaXoZRSHkVE9pxtmZ4aUkopH6dBoJRSPk6DQCmlfJwGgVJK+TgNAqWU8nEaBEop5eN8LgjW7ztKek6h3WUopZTb8Kkg2Hckn5ETlzDuw9V2l6KUUm7Dp4Jg2c5MANbsPcrh3OM2V6OUUu7Bp4Lg4S82nHz985bfbKxEKaXch2VBICLviki6iPx6luUiIm+ISIqIbBCRLlbVAlB4ouSU6WMFxVbuTimlPIaVRwTTgKEVLB8GJDh/xgJvWlgLWw/lAPDm9V3w9xOOFhRZuTullPIYlgWBMWYhcKSCVUYCHxiH5UBNEalvVT2/7s8GoF1cFJEhAWQXnLBqV0op5VHsvEYQB+wrM53mnGeJxrVDubJrQxrWqkFUjUA+Wr6XKQt3WrU7pZTyGHZ2Qy3lzDPlrigyFsfpIxo3bnxeO+ubGEPfREdX3F0a12J3Zj4vzNlK7vES7h+cgEh55SillPez84ggDWhUZrohcKC8FY0xU4wxScaYpJiYcsdVqJKbLmx68vUbP+8g/rE5vPbj9j/8vkop5YnsDIKZwE3Ou4d6AtnGmIOu2HGnRjV5+YoOp8x7b8muctctLTU88vkGhv9rEflFeqeRUsr7WHZqSEQ+BvoD0SKSBjwFBAIYY94C5gDDgRQgHxhjVS3lGd21IYfzjrMk5TBLUjJJrBtxctnmA8f49UA2VyU14qMVe/h0leNSxsLtGQxtZ9n1bKWUsoVlQWCMufYcyw1wt1X7Pxc/P+Gu/i24q38L7vxoNTvSc08uG/GfxRSXGg4cLWBJyuGT89OyCuwoVSmlLOVxYxZboWZo4Cm3kxaXOq5Zv/7TDgBu7NmEX7am82nyPo7kFXFx23p0bFTTllqVUqq6+VQXE2dTKzSIrLwiDmUXcuVbS89YflnHBozt24wd6blMmr+TV+Zus6FKpZSyhgYB0D2+NsWlhp4v/kzy7iwGtYply7NDadsgkquTGtE9vjaXd4mje9PaACxLzWTfkXybq1ZKqeqhp4aAC5rXOfn6gSGJ3DMoAYDZ9/bBcSkDIkICmTHuAnZm5DLoHwtYtOMw1/U4v2calFLKnegRARAc4M/Tl7WhS+OaZ3y5n/6gWbPoMKLDg1i5K/NkSCillCfTIHC6pVc8X97VizrhwRWuJyL0iK/D1+sOMH3lXgCKS0pJ3n1Eg0Ep5ZE0CM7D/UMSAZixKo29mfmMmZbMlW8t49sNLnkeTimlqpUGwXloERvO2L7NWL/vKH1fmceiHY5nDZbsOHzKepm5x8nUkdCUUm5OLxafp/EDW7B052Giw4OJq1mDJSmHWb03C2PMyesK1769nO2/5TLrnt60rh8JgL+fdm6nlHIvGgTnKTIkkFn39Dk5/eHyPfzf17/yxZr9jO7akANHC9j+m+Np5Uv/vRiAupHBfHtPb2IjQmypWSmlyqOnhqrJiI4NAHhhzhZyCk/ww6ZDZ6zz27HjvLOo/M7tlFLKLnpEUE2iagTywa3duendlbR/+gcAEuuG8+Ll7TleXErtsCAmzdvJB8v20D2+Nn0TYwj01xxWStlPv4mqUd/EGG6+oMnJ6QlDEunapDYXNo+mVb1I7h7QguLSUm57fxUPf77BxkqVUup/NAiq2TMj2/Hp2J6M69ecwa3rnrKsZb0IvruvD30Sovlq7X6W7jx8lndRSinX0SCwQI9mdXh0WCsCyjn10yI2grdvSiKuZg3+qaOiKaXcgAaBDUIC/bmhZxOSd2exMyP33BsopZSFNAhsckXXOAAG/WMBB7N1wBullH00CGwSGxHCtd0dHdy98r2Ob6CUso/ePmqjF0a1QwQ+dnZe9+jwVvqwmVLK5TQIbCQi/N8lbcguOMGXa/ezLDWTr+7qRb0oDQOllOvoqSGb1QjyZ+J1XXjo4pYczC6k10u/2F2SUsrHaBC4ieuc1wtKSg2HsgttrkYp5Us0CNxErbAg5j/Yn0B/4Yo3l+ptpUopl9EgcCNNo8N4ZGgr9h8t4I4PV1NaqiOeKaWsp0HgZm7v04x/Xt2RlPRcVu3JsrscpZQP0CBwQ4Na18VPYOH2DLtLUUr5AA0CNxQZEkjPZnWYvnIvRcWldpejlPJyGgRuakyveI7kFfHIF9pdtVLKWhoEbmpIm7pc0aUh36zbz+7DeXaXo5TyYhoEbuyRYS0J8Pfj3k/W6h1ESinLaBC4sdiIEJ4Y3poNadn8sPk3u8tRSnkpDQI3d12PxsRHh3Hnf1fz9sJUu8tRSnkhDQI3F+jvx2fjLqBbk9r8/futrN931O6SlFJeRoPAA0SHBzP1liRiI4K55+O1rNp9xO6SlFJeRIPAQ0SGBPLK6I7sPZLP9VNXsEvvJFJKVRMNAg/SOyGaWff0JtDfjwGvziclXTumU0r9cZYGgYgMFZFtIpIiIo+Ws7yxiMwTkbUiskFEhltZjzdoFxfFy6M7APD87M02V6OU8gaWBYGI+AMTgWFAG+BaEWlz2mp/BWYYYzoD1wCTrKrHmwxvX587+jVj3rYMXv9pu93lKKU8nJVHBN2BFGNMqjGmCPgEGHnaOgaIdL6OAg5YWI9X+cugRABe/2kHWXlFNlejlPJkVgZBHLCvzHSac15ZTwM3iEgaMAe4p7w3EpGxIrJKRFZlZGiPnOAY4nLWPb0RgX/qUYFS6g+wMgiknHmn95NwLTDNGNMQGA58KCJn1GSMmWKMSTLGJMXExFhQqmdqFxfFNd0a8UnyPvYdybe7HKWUh7IyCNKARmWmG3LmqZ/bgBkAxphlQAgQbWFNXmds3+YE+fvx1MxNdpeilPJQVgZBMpAgIvEiEoTjYvDM09bZCwwCEJHWOIJAz/1UQXx0GHcPaMEvW9NZrSOaKaXOg2VBYIwpBsYDc4EtOO4O2iQiz4rICOdqDwB/FpH1wMfALcYY7Waziq7r3pjQIH+unbKcSfNT7C5HKeVhxNO+d5OSksyqVavsLsPtbDuUw0vfb+WXren0bFabKTclERkSaHdZSik3ISKrjTFJ5S3TJ4u9RMt6Ebx5Qxd6t4hmeeoR7v14LZ4W8kope2gQeJHgAH8+ur0HD16UyPxtGTzw2XpyCk/YXZZSys0F2F2Aqn539m/BgexCpq/YS2pGHrf3ieeS9vURKe+OXqWUr9MjAi/k7ye8MKo9D13cknX7jjJ++lqemrmJ48UldpemlHJDekTgxe7q35zmMeF8mryXD5btYf62DL6+uxe1w4LsLk0p5Ub0iMCLiQhD29XjvTHdefCiRPYeyefGd1awPDXT7tKUUm5Eg8BHjB+YwJQbu7InM59rpixnysKddpeklHITGgQ+5KK29Vjy6EB6tajDC3O28uy3m0lJz9XbTJXycRoEPiaqRiDv3NyNzo1r8u6SXQx+bQHjp6+ltFTDQClfpReLfVBIoD+fjO3J8tQjzN+WzntLdnMwu4CXR3ekRWy43eUppVxMg8BHBQf40y8xhr4J0RgD05buZvBrCwgPDuCmC5pw3+AEggP87S5TKeUCGgQ+TkR46rI23Norns/XpLF2bxaT5u9kZ0Yuk67vir+fPoSmlLfTTufUGSbOS+GVudtoUieUG3s24bbe8fpUslIeTjudU1Vy94AWvHl9F+pGhvDc7C0Mfm0B01fspfCEPpmslDfSIFDlGta+Pp+O7cld/ZuzMyOPx7/ayLiPVrP9txy7S1NKVTMNAnVWIsLDQ1ux6ZmLuW9QAsm7jjDiP4tZsF0HkVPKm2gQqHMKCw7g/iGJzHuwP3E1a3Dzuyu5fupyth46ZndpSqlqoEGgKi02MoTZ9/bh8eGt2JCWzdDXF3HHh6vYmZGrD6Qp5cH0riF1XtKPFTJt6W4+WLaH3OPFNKxVg8eHt2Zo23r46S2nSrmdiu4a0iBQf8jB7AJmrjvAZ6vTSEnPJa5mDX64vy9hwfqIilLuRINAWa6k1HDvx2uZvfEgtUIDubB5NAl1w7lnYII+lKaUG9DnCJTl/P2E/1zXmck3dqVNg0gWbM/g9Z92MPYDxzUEpZT70iMCZZmpi1L554/bCQ0OYOpNSbSsF0FIoPZfpJQd9IhA2eL2Ps2Y/ueeZOQcZ+TEJQx+bQEz1x8gM/e43aUppcrQK3rKUh0b1eS9Md2Ytf4ga/Zmce/HawG4d1ACozrHER8dZnOFSik9NaRcpqCohM/XpPHUN79SaiA4wI/v7utDsxgdA0Epq+mpIeUWagT5c2PPJvw4oR9v35REkL8ft05LJiU9l+KSUrvLU8pnaRAol2seE86QNnV5++Yk9h7JZ/BrC7jizaVk5RXZXZpSPkmDQNmmZ7M6fHlXL265sCkb9mdzzZTlHM3XMFDK1TQIlK06NarJ0yPaMvG6LuxIz6HTsz/y0vdb8bRrV0p5Mg0C5RaGt6/PxOu6MLh1Xd6cv5O3F6XaXZJSPkODQLmNYe3r8/ZNXRnWrh4vfreVpTsP212SUj5Bg0C5FRHhpdEdiAgO4Lq3V7Bu31G7S1LK62kQKLcTGRLI36/oAMDlk5Ywb2u6zRUp5d00CJRbGt6+Pgse6k/T6DDu+Xgt3/96SG8vVcoilgaBiAwVkW0ikiIij55lnatEZLOIbBKR6VbWozxLkzph/Pf2HoQF+zPuo9Vc9PpCDmUX2l2WUl6nUkEgIveJSKQ4vCMia0TkonNs4w9MBIYBbYBrRaTNaeskAI8BvYwxbYG/nNdvobxW/agafHRbD8b0asrR/CKen7PF7pKU8jqVPSK41RhzDLgIiAHGAH8/xzbdgRRjTKoxpgj4BBh52jp/BiYaY7IAjDF6MlidIaFuBE9d1pZRneP4dv0Bftr8m90lKeVVKhsEvw8xNRx4zxizvsy8s4kD9pWZTnPOKysRSBSRJSKyXESGVrIe5YP+emkbmtYJ5clvfmXWhgP60JlS1aSyQbBaRH7AEQRzRSQCOFcvYeUFxen/cgOABKA/cC0wVURqnvFGImNFZJWIrMrIyKhkycrbRIYE8uqVHQkJ9Gf89LV8uWa/3SUp5RUqGwS3AY8C3Ywx+UAgjtNDFUkDGpWZbggcKGedb4wxJ4wxu4BtOILhFMaYKcaYJGNMUkxMTCVLVt4oqWltfpzQj44No3j8q42sSM20uySlPF5lg+ACYJsx5qiI3AD8Fcg+xzbJQIKIxItIEHANMPO0db4GBgCISDSOU0Xat4CqkGN85C7UCQvi6inLGTVpCbsP59ldllIeq7JB8CaQLyIdgYeBPcAHFW1gjCkGxgNzgS3ADGPMJhF5VkRGOFebC2SKyGZgHvCQMUb/xFPn1Kh2KJ/ecQFXJTVk7d6jDP3XQjJydAhMpc5HpUYoE5E1xpguIvIksN8Y887v86wv8VQ6QpkqyxjDV2v3M2HGeqJqBLLgof7UDA2yuyyl3E51jFCWIyKPATcCs53PCARWV4FKnS8R4fIuDXlmRFuyC04w4NX5LE3RzuqUqorKBsHVwHEczxMcwnEb6CuWVaVUFd18YVM+GduTUgPXTV3BDVNX8O7iXRQUldhdmlJur1JB4Pzy/y8QJSKXAoXGmAqvESjlaj2b1eGFUe0BWJxymGdnbebNBTttrkop91fZLiauAlYCVwJXAStEZLSVhSl1Pi7pUJ91Tw7hs3EX0DwmjDd+3sHPW/RJZKUqUtlTQ0/geIbgZmPMTTi6j/g/68pS6vzVDA2iW9PajB/YAoDb3l/FwewCm6tS6o9Zt+8opaXWPE1f2SDwO60foMwqbKuULUZ1bsj023sA8PTMTTZXo9T523U4j8snLWGKRUO4BlRyve9FZC7wsXP6amCOJRUpVY0ubBHNrb3i+WjFHo7mF+mtpcojTV6wkwB/Py7vcnp3bdWjsheLHwKmAB2AjsAUY8wjllSkVDUb1TkOYwwPzFhvdylKVdmh7EK+WJPGVUkNiY0IsWQflT0iwBjzBfCFJVUoZaH2DaO4q38L3vhlBweOFtCgZg27S1Kq0qYuSqXUwB19m1u2jwqPCEQkR0SOlfOTIyLHLKtKqWp2RZeGGAMPzFhPTuEJu8tRqlKy8oqYvnIvl3WoT6PaoZbtp8IgMMZEGGMiy/mJMMZEWlaVUtWscZ1QJgxJZFlqJu8u3m13OUpVyvvLdpNfVMKd/VtYuh+980f5jHsHJdC5cU2+XJtGUfG5htNQyl55x4uZtnQ3g1vH0rJehKX70iBQPuX23s3Yk5nP0p3aH5Fybx+v3MvR/BOWHw2ABoHyMYNax1I7LIj7P12n3VYrt1VcUsrURbvoEV+brk1qWb4/DQLlU0IC/Xn96k5k5Z+gxws/cUwvHCs3tCM9l0PHCrmme6Nzr1wNNAiUz+mbGMNfBidQauCL1Wl2l6PUGTakHQWgUyPrjwZAg0D5qL8MTqRrk1pMWZjK8WLtqlq5l/Vp2USEBNDEwltGy9IgUD5rXL/mHMwuZOWuI3aXotRJxhgWbs8gqUkt/PzEJfvUIFA+q3eLaIID/Phhk3ZTrdzHmr1HScsq4NIODVy2Tw0C5bNqBPlzcdt6fLh8j45ZoNzGzHX7CQ7w46K2dV22Tw0C5dPuHfS/MQus6utdqcoqLill9saDDGodS0SI64aF1yBQPq1FbASjOju69n3w8/UYo2Gg7LN0ZyaHc4sY0dF1p4VAg0Apnh/VjriaNfhyzX7iH5vD3E2H7C5J+aiZ6w8QERxA/5axLt2vBoHyeaFBASx6eADX9WgMwB0frqb9U3OZMGMdhSf01lLlGsYYFmzPoF/LGEIC/V26bw0CpQA/P+GFUe15+6YkAHKOF/Plmv088JmeLlKusf23XDJyjtMnIdrl+670wDRK+YIhbeqy8vFBHC8uZdxHq5m94SC1QgMZ0TGOVvUjiHThBTzlWxanODpC7J0Q4/J96xGBUqeJjQyhUe1QZo7vTe8W0Xy0fC9XTV7GBS/8THpOod3lKS+1eEcG8dFhxNkwgp4GgVJn4e8nTBvTjZeuaM/FbeuSV1TCjOR9dpelvFBRcSkrdh2hdwvXnxYCDQKlKhTg78fV3Roz+cYkusfX5su1+/Wagap2a/dmkV9UQm8brg+ABoFSlXZph/qkZuSxOzPf7lKUl1mcchh/P+GC5nVs2b8GgVKV1Nd5EW/h9gybK1HeZtnOTNrHRdl2M4IGgVKV1DQ6jBax4by/bLc+X6CqTeGJEjakZdMjvrZtNWgQKFUF/3dpG1Iz8pg4L8XuUpSXWLfvKEUlpXRrqkGglEfolxjDnzo1YPLCVLLyiuwuR3mB5F1HEEGDQClPMq5/c4qKS/lEbyVV1WDl7iO0rBtBVKh9DytqEChVRa3qRdInIZo356dQXFJqdznKgxWXlLJmT5atRwNgcRCIyFAR2SYiKSLyaAXrjRYRIyJJVtajVHW5MqkRxwqLWZ6qw1yq87f54DHyikroZuOFYrAwCETEH5gIDAPaANeKSJty1osA7gVWWFWLUtVtYKtYYiKCeWdxqt2lKA/2+3jZ3b34iKA7kGKMSTXGFAGfACPLWe9vwMuAduKiPEZ4cAAjOjZgSUomOYUn7C5HeaiVu47QuHYo9aJCbK3DyiCIA8peTUtzzjtJRDoDjYwxsyp6IxEZKyKrRGRVRoY+zKPcw7B29SgqKeWXrel2l6I81Go3uD4A1gaBlDPvZCctIuIH/BN44FxvZIyZYoxJMsYkxcS4votWpcrTpXEtaocFcd8n68g9Xmx3OcrDFBSVkJlXRPPYMLtLsTQI0oBGZaYbAgfKTEcA7YD5IrIb6AnM1AvGylP4+QlXdHEc5I5+c6l2Ua2qJCvf8RxKrdAgmyuxNgiSgQQRiReRIOAaYObvC40x2caYaGNMU2NMU2A5MMIYs8rCmpSqVo8Na83LozuQkp7Lv37aYXc5yoP8LwjsH+zIsiAwxhQD44G5wBZghjFmk4g8KyIjrNqvUq7k5ydcldSI4e3r898Ve9mQdtTukpSHyMpz3GTg7UcEGGPmGGMSjTHNjTHPO+c9aYyZWc66/fVoQHmqv17amvDgAJ6auUnHK1CV8tsxx6nE6IhgmyvRJ4uVqhaxESE8Prw1a/ceZdnOTLvLUR5gZ0YuAX5C49qhdpeiQaBUdbm8SxyxEcGMmZbM/G3pemSgzqqk1LA45TAtYsMJ9Lf/a9j+CpTyEiGB/tw/JJHjxaXc8l4y8Y/N4cs1aRoI6gxfrEljQ1o21/VobHcpgAaBUtXqT51OeWaSCTPWM3P9gbOsrXzRiZJSXv9xOy3rRnBjzyZ2lwNAgN0FKOVNagT5s+P5YRgDP27+jZfnbuXFOVtpFxdFnbAgarrBHSLKXot2ZHAgu5ApI9oiUt5zt66nQaBUNfv9nO8lHeoTERLATe+uZNA/FhBVI5A3r+/CG7/soG5kCP+6prPNlSo7zNpwkJqhgfRvGWt3KSfpqSGlLNQnIZpbe8UTERJAdsEJrpu6guWpR/hm3QHmbjpkd3nKxYwxLNuZSa8W0QQFuM/Xr/tUopQXEhGevKwNG5++mEUPD6BFbDhXJzWibYNInvhqo/Zc6mNS0nM5mF1IT5vHHzidBoFSLtKodig/TejHS6M78MTw1hzOLeIfP2y3uyzlIoUnSrjlvWRCg/zd6rQQaBAoZYsLW0Rzeec4pi3dzewNB/UWUx/w5Zr97D9awFs3dKWRGzxEVpYGgVI2eWZkW+Jq1uDu6Wt4auYmu8tRFpu3LZ2mdULpkxBtdyln0CBQyiYRIYF8ekdPaocF8cGyPXrx2MttO5RD2wZRbnPLaFkaBErZqGGtUFY8PojEuuH8/butOsCNl1q/7yh7j+TTpUktu0splwaBUjYL9Pfjr5e0YXdmHq98v9XucpQFpi7eRWRIAKM6x517ZRtoECjlBvomxnB9j8a8v2wPczYetLscVc3W7MmiT2IMtcPc88lyDQKl3MTTl7WlVb0IXpm7jePFJXaXo6pJek4h+48W0LlRTbtLOSsNAqXcRIC/HxOGJLLrcB4t//o9WXlFdpekqsHy1CMAdG7sntcHQINAKbcypE1dBrZyPGz0/rLdttaiqiY7/wSFJ0o4UVJ68rmQFamZ3PvxWqLDg+nkxkcE2umcUm5ERHj3lm7c/n4y//4lhZGd4oiPDrO7LFUJ3V/4iePFpeUue/jilvj7ud9to7/TIwKl3NCLl3cgyN+PR7/YQJ7eUur2SkvNyRC4oFmdU5ZFhwdzZVJDO8qqND0iUMoNxUQE8/yodkyYsZ5Rk5bw/X198XPjvyh92Zq9Wby9MBWApy9rwy294sk7XkxaVgELt2cwpE1dt3yIrCwNAqXc1OVdGjJrw0F+2ZrOgu0ZDGjlXh2VKdibmc+Vby2jpNRxTSChbgQAYcEBtKwXQct6EXaWV2l6akgpN/bWDV2pGxnMlIWp2jGdG9h3JJ93F++i1PnFn5KRQ0mp4a+XtGbZYwO5sHmdc7yDe9IgUMqNBQX48ec+zViWmsl7S3bbXY5PKy4pZcKMdTw7a/PJO7r2Hy0E4LKODagfVcPtTwGdjQaBUm7utt7xDGwVy/NztpCWlW93OT7rk+R9JO/OAuDFOVv5/teDbD5wjNAgf2LCg22u7o/RIFDKzYkIf/tTO4wxDHh1Pt+s2293ST5p26EcAH6a0I9W9SMY99EaPl65lwEtYz3+Qr4GgVIeIK5mDR4b1pqSUsN9n6yj6aOzmb3hoHZF4UJpWfm0rh9Ji9hw/nVN55Pzx/VrbmNV1UODQCkP8ee+zUh+YvDJ6bunr+GNn3fYWJHvOFZ4ghW7jtC6vuMuoPjoMFKeH8buv19C+4ZRNlf3x2kQKOVB6oQHs+LxQdw3KAF/P2HivJ38tPk3u8vyehv2ZZNfVHJKN9IB/t7z9ek9v4lSPqJuZAj3D0nk5wn9ALj9g1Vk5h63uSrvNnvjAfwE2sd5/l//5dEgUMpDNY0OY+J1XQB4Z/Eum6upflMXpZL03I+MnLiEwzYG3Tfr9vPxyn2M6RVPzVD3HE/gj9IgUMqDXdKhPpd2qM+k+TuZty3d7nKqhTGGrYeO8dzsLRzOLWL9vqM88+1m2+r5aPkeGtcO5ZGhrWyrwWoaBEp5uJeu6ECz6DDGvJfMwu0Zdpfzhz38+QaGvr4IgJ8f6McdfZvx7foDvLfE9Uc9+47kk7w7i6u7NSIowHu/Lr33N1PKR4QFBzDphi5E1Qjk2VmbT/Z744nyi4r5bHUaAKM6x9E8Jpz7hyTSJyGaZ77dzIa0oy6tZ/LCnYjAiI4NXLpfV9MgUMoLtKoXyfOj2pGSnsvXa8t/4MzdA2JjWjZXT14OwIuXt+fl0R0ACAn059UrOxIW5M/Dn2842c+P1YwxzNl4iEs7NKBR7VCX7NMuGgRKeYlh7erTPi6Kp7/dRHbBiVOWpaTn0OVvPzLwH/O5fupypq/YS97xYp785leX/5VdntJSw9gPV7FxfzYjOjbgii4NCSxze2bdyBCeG9WOrYdymLvpEI99uZHLJy3hYHaBZTXtzMjlSF4RfVpEW7YPd2FpEIjIUBHZJiIpIvJoOcsniMhmEdkgIj+LSBMr61HKm/n7CS9e3p6cwmImzks5ZdmnyfvILjhBakYeS1IyefyrjVz51jI+WLaHEf9Zwq3Tklm847BNlcOy1EwOZhfy6LBWvHFt53LPx4/oGEez6DDu/K+ja4c1e49y0T8XsvtwniU1fbbKcYqqe3xtS97fnVgWBCLiD0wEhgFtgGtFpM1pq60FkowxHYDPgZetqkcpX9AuLorBrRaXMNEAAA7BSURBVGOZsjCVyQt2Ao67Xt5etIt+iTHsenE4X911IQ2iQth88Bi1wxy3Q/6yNZ0b3lnBvG3pLu/u+ljhCR79cgMxEcHccmHTs67n7yeMH9ji5PSrV3bkREnpGaH3R5SUGpbuPMzURalMXexos6Y+MFSolQPTdAdSjDGpACLyCTASOHkfmDFmXpn1lwM3WFiPUj7htas70e25n3jxu618tjqNlPRcAK7r0RgRoXPjWnx3X19+2HyIkZ3i+GXrb/y0JZ3PV6cx5r1knvtTO27o6bqD89kbDrLvSAEf3daDkED/Ctcd0bEBf/9uK/lFJYzo2IBf92czbeluggP9+NvIdufdDXRqRi5XTV5+xvMKg9vUPa/38zRWBkEcsK/MdBrQo4L1bwO+K2+BiIwFxgI0bty4uupTyitFhgTy04R+XDV5GSnpuYQHBzD15iR6lhlLNyo0kCuTGgEwtF19hrarz8BWsdz13zX888ft/KlzHOHBrhnAcOvBY4QF+dOrxbkHdQnw92Peg/3JPV5MUIAfjw5rRakxfLBsD7ERIVyV1Ih6USFnbFdaasrtITQ9p5AHP9twym23PeJrM6RNXfz9hGu6Nfpjv5yHEKsOA0XkSuBiY8ztzukbge7GmHvKWfcGYDzQzxhT4SOESUlJZtWqVVaUrJRXKS01iFClv5LX7s1i1KSljOzU4JQeNq107ZTlFJwo4eu7e53X9kXFpVzyxiJ2pOcS5O/H93/pQ3hwwMkjoamLd7E7M48v77yQiJBA/J2BkJaVT5+X52EMhAcHMG1MN+pGhnjtHUIistoYk1TeMisjPw0oG6cNgQOnryQig4EnqEQIKKUq73z6yO/cuBZ39GvG5AWp/LlPM9q5oG+dHek5DPwD4zEHBfjxzfhefL46jSe/2cTAfywod71Oz/548vV7t3RjzLRkAB66uCV39W/usaOLVQcr7xpKBhJEJF5EgoBrgJllVxCRzsBkYIQxxjuej1fKw43r25w6YUGMmZZseWd2mbnHOZxbRGLdPzbIe2hQADdd0JT3b+1OrdBAggL8qOG83hAZEsDdA04dM+D3EAC4e0ALnw4BsPCIwBhTLCLjgbmAP/CuMWaTiDwLrDLGzAReAcKBz5z/I/YaY0ZYVZNS6txqhQXx0hUduP2DVTz0+Qbevinp5OmU6rbtN8eoXwl/MAh+1y8xhmWPDcIYqBHkjzHm5Jf8PQMTyDteTHrOcV77cTsBflLhXUq+xLJrBFbRawRKucbURak8N3sLd/Rrxp39muPnJ0SGBFbb+5eUGi55YxG7M/NY8shA6nj4uL/uzq5rBEopD3Zb73jWp2UzeUEqkxekUicsiJ8m9KNW2Nm7Yi4oKiEk0K9Sp1r2ZxWw9VAOz4xoqyFgM+1iQilVLhHhjWs60b2p48nazLwi7v1k7RlPIB8vLmHivBQ+Wr6Hrs/9SPunf+CDZbspLill+285Z+0bKCXDcVqoXVykpb+HOjc9IlBKnZWI8N6YbqSk57I45TCvzN3Goh2HeeuGrgxqHcuJklKue3sF6/b9r7+iQH/hyW828eQ3mwC4sWcT/vandme89850R9cQzaLDXfPLqLPSIFBKVSgsOICOjWrSsVFNhrarx5VvLWPcR6uJDg/igubRJ0Pg2ZFt6ZsQQ1ytGrw5fyebDjjG+f1w+R66NKnJqM4NT76nMYZZGw9SJyyowlNNyjU0CJRSldY8JpxfHujH0zM38fW6A3y7/gD9W8bw6pUdiS5znv/eQQmA44Lw1ZOX8cjnG4mqEcjAVo4uG1bsOsL6fUe9vp9/T6HXCJRSVVIzNIjXr+nM48MdQzeO7trwlBAoy99PePumJJrHhnPrtFV8t/Eg+UXFjJ++loa1avD8qDNPGSnX09tHlVLnLSU9h+Yx4ee8S2jzgWMMf2MR/n5CjUB/co8X8+nYnvRodu7+hVT10NtHlVKWaBFbuQfB2jSIZPEjA/j3zylkF5xgZKcGGgJuRINAKeUSDWuF8pJz+EnlXvQagVJK+TgNAqWU8nEaBEop5eM0CJRSysdpECillI/TIFBKKR+nQaCUUj5Og0AppXycx3UxISLZwI7TZkcB2eW8BogGDmON0/dVndtUtN7ZllV2fkXT7tZeld3uXOtUpc2q0l7gfm3m6s9YVdvQ3dqrstt5+mesiTEmptwlxhiP+gGmVDTv9OU4xkd2WS3VtU1F651tWWXnVzTtbu1V2e3OtU5V2qwq7eWObebqz1hV29Dd2quy23nrZ8wY45Gnhr49x7zyllvlfPZV2W0qWu9syyo7/1zTVjnf/VRmu3OtU5U2c5f2Ot99ufozdj5taBX9jJ0Hjzs1VFUissqcpcc9dSZtr6rTNqsaba+qs7rNPPGIoKqm2F2Ah9H2qjpts6rR9qo6S9vM648IlFJKVcwXjgiUUkpVQINAKaV8nAaBUkr5OJ8OAhHpIyJvichUEVlqdz3uTkT8ROR5Efm3iNxsdz2eQET6i8gi5+esv931eAIRCROR1SJyqd21uDsRae38bH0uInee7/t4bBCIyLsiki4iv542f6iIbBORFBF5tKL3MMYsMsaMA2YB71tZr92qo72AkUAccAJIs6pWd1FNbWaAXCAEL2+zamovgEeAGdZU6T6q6Ttsi/M77CrgvG8v9di7hkSkL45/YB8YY9o55/kD24EhOP7RJQPXAv7Ai6e9xa3GmHTndjOA240xx1xUvstVR3s5f7KMMZNF5HNjzGhX1W+Hamqzw8aYUhGpC7xmjLneVfW7WjW1Vwcc3SmE4Gi7Wa6p3vWq6ztMREYAjwL/McZMP59aPHbwemPMQhFpetrs7kCKMSYVQEQ+AUYaY14Eyj3MFJHGQLY3hwBUT3uJSBpQ5Jwssa5a91BdnzGnLCDYijrdRTV9xgYAYUAboEBE5hhjSi0t3CbV9fkyxswEZorIbMC3guAs4oB9ZabTgB7n2OY24D3LKnJvVW2vL4F/i0gfYKGVhbmxKrWZiFwOXAzUBP5jbWluqUrtZYx5AkBEbsF5NGVpde6nqp+v/sDlOP7ImHO+O/W2IJBy5lV47ssY85RFtXiCKrWXMSYfR3D6sqq22Zc4AtRXVfnfJIAxZlr1l+IRqvr5mg/M/6M79diLxWeRBjQqM90QOGBTLZ5A26vqtM2qRturamxpL28LgmQgQUTiRSQIuAaYaXNN7kzbq+q0zapG26tqbGkvjw0CEfkYWAa0FJE0EbnNGFMMjAfmAluAGcaYTXbW6S60vapO26xqtL2qxp3ay2NvH1VKKVU9PPaIQCmlVPXQIFBKKR+nQaCUUj5Og0AppXycBoFSSvk4DQKllPJxGgTKciKS64J9jKhkF8fVuc/+InLheWzXWUSmOl/fIiJu0QeRiDQ9vUvkctaJEZHvXVWTcg0NAuUxnF30lssYM9MY83cL9llRf1z9gSoHAfA48O/zKshmxpgM4KCI9LK7FlV9NAiUS4nIQyKSLCIbROSZMvO/do5KtUlExpaZnysiz4rICuACEdktIs+IyBoR2SgirZzrnfzLWkSmicgbIrJURFJFZLRzvp+ITHLuY5aIzPl92Wk1zheRF0RkAXCfiFwmIitEZK2I/CQidZ3dB48D7heRdeIY7S5GRL5w/n7J5X1ZikgE0MEYs76cZU1E5Gdn2/zs7CIdEWkuIsud7/lseUdY4hjVa7aIrBeRX0Xkauf8bs52WC8iK0UkwvmX/yJnG64p76hGRPxF5JUy/6/uKLP4a8Brx1XwScYY/dEfS3+AXOd/LwKm4Ohh0Q/HyHB9nctqO/9bA/gVqOOcNsBVZd5rN3CP8/VdwFTn61twDMwBMA34zLmPNjj6dwcYjaOrXj+gHo4xAkaXU+98YFKZ6Vr87yn824F/OF8/DTxYZr3pQG/n68bAlnLeewDwRZnpsnV/C9zsfH0r8LXz9SzgWufrcb+352nvewXwdpnpKCAISAW6OedF4uhxOBQIcc5LAFY5XzcFfnW+Hgv81fk6GFgFxDun44CNdn+u9Kf6frytG2rl3i5y/qx1Tofj+CJaCNwrIqOc8xs552fiGADni9Pe5/dunVfj6Iu9PF8bR1/2m8UxOhhAb+Az5/xDIjKvglo/LfO6IfCpiNTH8eW66yzbDAbaiJzsSThSRCKMMTll1qkPZJxl+wvK/D4fAi+Xmf8n5+vpwKvlbLsReFVEXgJmGWMWiUh74KAxJhnAOAdfEpEw4D8i0glH+yaW834XAR3KHDFF4fh/sgtIBxqc5XdQHkiDQLmSAC8aYyafMtMxuMZg4AJjTL6IzMcxVCFAoTHm9NHQjjv/W8LZP8PHy7yW0/5bGXllXv8bxzCTM521Pn2Wbfxw/A4FFbxvAf/73c6l0h2BGWO2i0hXYDjwooj8gOMUTnnvcT/wG9DRWXNhOesIjiOvueUsC8HxeygvodcIlCvNBW4VkXAAEYkTkVgcf21mOUOgFdDTov0vBq5wXiuoi+Nib2VEAfudr28uMz8HiCgz/QOOniMBcP7FfbotQIuz7Gcpjm6HwXEOfrHz9XIcp34os/wUItIAyDfGfITjiKELsBVoICLdnOtEOC9+R+E4UigFbsQxHu7p5gJ3ikigc9tE55EEOI4gKry7SHkWDQLlMsaYH3Cc2lgmIhuBz3F8kX4PBIjIBuBvOL74rPAFjoE/fgUmAyuA7Eps9zTwmYgsAg6Xmf8tMOr3i8XAvUCS8+LqZhzn809hjNkKRDkvGp/uXmCMsx1uBO5zzv8LMEFEVuI4tVReze2BlSKyDngCeM4YUwRcjWN40fXAjzj+mp8E3Cwiy3F8qeeV835Tgc3AGuctpZP539HXAGB2OdsoD6XdUCufIiLhxphcEakDrAR6GWMOubiG+4EcY8zUSq4fChQYY4yIXIPjwvFIS4usuJ6FOAZUz7KrBlW99BqB8jWzRKQmjou+f3N1CDi9CVxZhfW74ri4K8BRHHcU2UJEYnBcL9EQ8CJ6RKCUUj5OrxEopZSP0yBQSikfp0GglFI+ToNAKaV8nAaBUkr5OA0CpZTycf8PM5NE9u6LuWcAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learner.lr_plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "begin training using onecycle policy with max lr of 2e-05...\n",
      "Train on 1788 samples, validate on 447 samples\n",
      "Epoch 1/5\n",
      "1788/1788 [==============================] - 40s 22ms/sample - loss: 0.7808 - acc: 0.5347 - val_loss: 0.6646 - val_acc: 0.6421\n",
      "Epoch 2/5\n",
      "1788/1788 [==============================] - 37s 21ms/sample - loss: 0.6231 - acc: 0.6790 - val_loss: 0.5366 - val_acc: 0.7338\n",
      "Epoch 3/5\n",
      "1788/1788 [==============================] - 37s 21ms/sample - loss: 0.4582 - acc: 0.7707 - val_loss: 0.4538 - val_acc: 0.8009\n",
      "Epoch 4/5\n",
      "1788/1788 [==============================] - 37s 21ms/sample - loss: 0.3046 - acc: 0.8758 - val_loss: 0.4647 - val_acc: 0.7964\n",
      "Epoch 5/5\n",
      "1788/1788 [==============================] - 37s 21ms/sample - loss: 0.1672 - acc: 0.9418 - val_loss: 0.4650 - val_acc: 0.8188\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7f2eb729ff90>"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 2e-5 is one of the LRs  recommended by Google and is consistent with the plot above.\n",
    "learner.fit_onecycle(2e-5, 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictor = ktrain.get_predictor(learner.model, preproc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "pred = predictor.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "slicing_num = len(pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "count = 0\n",
    "\n",
    "for i in range(slicing_num):\n",
    "#     print('pred:', pred[i])\n",
    "#     print('y_test', y_test[i])\n",
    "    if pred[i] == y_test[i]:\n",
    "        count = count + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8187919463087249"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "float(count / slicing_num)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
